const http = require('http');
const cheerio = require('cheerio');
const urlDecoder = require('./dbSource').urlDecoder;
const request = require('request');
const decodeXml = require('../core/dbSource').decodeXml;
const sited = require('../database/db').sited;
const DOMParser = require('xmldom').DOMParser;
const getTagContent = require('../core/dbSource').getTagContent;


request('http://sited.ka94.com/', function (error, response, body) {
    if (!error && response.statusCode === 200) {
        const $ = cheerio.load(body.toString());
        let list = $('li div a.button');
        const set = [];
        list.each(function (i, ele) {
            let url = urlDecoder(ele.attribs.href);
            console.log(url);
            set.push(url);
        });
        set.forEach(function (item) {

            request(item, function (err, res, bd) {
                if(!error && res.statusCode === 200) {
                    let xml = bd;
                    let msg = '';
                    if (xml.startsWith('<?xml')) {

                    }else if(xml.startsWith('sited::')) {
                        msg ='this is sited encode xml file';
                        console.log(msg);
                        xml = decodeXml(xml);
                    } else {
                        xml = null;
                    }

                    if(xml) {
                        // console.log(xml);
                        const doc = new DOMParser().parseFromString(xml);
                        let title = getTagContent(doc, 'title');
                        let guid = getTagContent(doc, 'guid');
                        let intro = getTagContent(doc, 'intro');
                        let author = getTagContent(doc, 'author');

                        let sited_obj = new sited({
                            title: title,
                            guid: guid,
                            intro: intro,
                            author: author,
                            xml: xml
                        });
                        sited_obj.save(function(err) {
                            if (err) {
                                console.log('保存失败');
                                console.log(err);
                                msg= '保存失败';
                                return;
                            }
                            console.log('保存成功');
                        });
                    }

                }
            })
        });
    }
});